/**
 * crawler
 *
 * outlook@China software studio
 * Copyright (c) 2008-2011. All Rights Reserved.
 * http://www.outlook.com
 */

package org.outlook.crawler.util;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;

import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.outlook.common.log.LogUtil;
import org.outlook.crawler.client.domain.Source;
import org.outlook.crawler.client.exception.EvaluateException;
import org.outlook.crawler.client.pojo.MethodType;
import org.outlook.crawler.client.pojo.Webinfo;
import org.outlook.crawler.util.parser.ELParser;
import org.outlook.crawler.util.parser.ScriptParser;

/**
 * function:
 * 
 * @author watson email: watson.wu@hotmail.com
 * @date 2011-4-26
 */
public class Fetcher {

	private static final String CHARSET = "charset";

	private static final String CONTENT_TYPE = "Content-Type";

	@SuppressWarnings("unused")
	private static final String CONTENT_LENGTH = "Content-Length";
	private static final String CONTENT_LOCATION = "Content-Location";

	private Log logger = LogFactory.getLog(Fetcher.class);

	public static final String IE7_USERAGENT = "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)";
	public static final String FF3_USERAGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3";
	private static final String DEFAULT_CHARSET = "utf-8";


	/**
	 * function: open connection & return inputStream
	 * 
	 * @param requestUrl
	 * @param conn
	 * @return
	 */
	public static InputStream getGetRequest(String requestUrl,
			URLConnection conn) {
		try {
			URL url = new URL(requestUrl);
			conn = url.openConnection();
			conn.connect();
			InputStream is = conn.getInputStream();
			return is;
		} catch (MalformedURLException e) {
			// TODO handle exception
		} catch (IOException e) {
			// TODO handle exception
		} finally {
		}
		return null;
	}

	/**
	 * function: simple get HTML sources
	 * 
	 * @param requestUrl
	 * @param encode
	 * @return
	 */
	public static String getHtmlSource(String requestUrl, String encode) {
		HttpURLConnection conn = null;
		InputStream inputStream = getGetRequest(requestUrl, conn);
		if (null != inputStream) {
			try {
				String htmlSource = IOUtils.toString(inputStream, encode);
				return htmlSource;
			} catch (IOException e) {
				// TODO handle exception
			} finally {
				try {
					inputStream.close();
					if (conn != null)
						conn.disconnect();
				} catch (IOException e) {
					// TODO handle exception
				}
			}
		}
		return null;
	}

	public Webinfo fetch(String url) {
		String htmlSource = getHtmlSource(url, "utf-8");
		try {
			if (null != htmlSource) {
				return new Webinfo(htmlSource, HttpStatus.SC_OK);
			} else
				return new Webinfo(null, HttpStatus.SC_BAD_REQUEST);
		} catch (EvaluateException e) {
			e.printStackTrace();
		}
		return null;
	}

	public Webinfo fetch(String url, Source source) {
		HttpClient httpClient = new HttpClient();
		HttpClientParams clientParams = httpClient.getParams();
		clientParams.setSoTimeout(30000);
		// clientParams.setBooleanParameter("http.protocol.expect-continue", false);
		HttpConnectionManagerParams connectionParams = httpClient.getHttpConnectionManager().getParams();

		connectionParams.setConnectionTimeout(30000);
		// connectionParams.setStaleCheckingEnabled(false);
		HttpMethod httpMethod = null;
		String requestURL = url;
		source.setRequestURL(requestURL);
		if (source.getType() ==null || source.getType() == MethodType.GET) {
			// requestURL = getGetUrl(url, resource);
			try {
				httpMethod = new GetMethod(url);
			} catch (Exception e) {
				LogUtil.error(logger, url, e.getMessage());
			}
		} else if (source.getType() == MethodType.POST) {
			httpMethod = new PostMethod(requestURL);
			if (StringUtils.isNotEmpty(source.getParamKeyValue())) {
				NameValuePair[] data = getParameters(source);
				((PostMethod) httpMethod).setRequestBody(data);
			}
		}
		if (null == httpMethod) {
			// TODO
			return null;
		}
		httpMethod.setFollowRedirects(source.isFollowRedirect());
		httpMethod.getParams().setSoTimeout(30000);

		httpMethod.addRequestHeader("User-Agent", FF3_USERAGENT);
		httpMethod.addRequestHeader("Connection", "close");

		httpMethod.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
		if (StringUtils.isNotEmpty(source.getCookies())) {
			Cookie[] cookies = getCookies(source);
			httpClient.getState().addCookies(cookies);
			// httpMethod.setRequestHeader("Cookie", cookie);
		}

		try {
			int httpStatus = httpClient.executeMethod(httpMethod);
//			for (Header header : httpMethod.getResponseHeaders()) {
//				LogUtil.debug(logger, header.getName(), " = ", header.getValue());
//			}
			if (httpStatus == HttpStatus.SC_OK) {
				byte[] bodybytes = httpMethod.getResponseBody();
				String htmlSource = getSourceInCharset(bodybytes, source, httpMethod);

				Webinfo webinfo = new Webinfo(htmlSource, httpStatus);

				Header contentTypeHeader = httpMethod.getResponseHeader(CONTENT_TYPE);
				if (contentTypeHeader!=null	&& StringUtils.isNotBlank(contentTypeHeader.getValue())) {
					String contentTypeStr = contentTypeHeader.getValue().toLowerCase();
					if (contentTypeStr.contains("text/html"))
						webinfo.setContentType("text/html");
				}
				
//				if(requestURL.equalsIgnoreCase(source.getUrl())) {
					if(httpMethod.getResponseHeader(CONTENT_LOCATION)!=null
							&& StringUtils.isNotBlank(httpMethod.getResponseHeader(CONTENT_LOCATION).getValue()))
						webinfo.setUrl(httpMethod.getResponseHeader(CONTENT_LOCATION).getValue());
				/*}*/ else
					webinfo.setUrl(requestURL);
				
				return webinfo;
			} else {
				// TODO retry/report?
				Webinfo webinfo = new Webinfo(null, httpStatus);
				return webinfo;
			}

		} catch (IOException e) {
			// TODO
			LogUtil.error(logger, e.getMessage());
		} catch (EvaluateException e) {
			// TODO
			LogUtil.error(logger, e.getMessage());
		} finally {
			if (httpMethod != null)
				httpMethod.releaseConnection();
		}

		return null;
	}

	private String getSourceInCharset(byte[] bodybytes, Source source, HttpMethod httpMethod) throws UnsupportedEncodingException {
		String charset = StringUtils.isNotBlank(source.getCharset()) ? source.getCharset() : DEFAULT_CHARSET;
		Header contentTypeHeader = httpMethod.getResponseHeader(CONTENT_TYPE);
		if (contentTypeHeader!=null && StringUtils.isNotBlank(contentTypeHeader.getValue())) {
			String contentTypeStr = contentTypeHeader.getValue().toLowerCase();
			Map<String, String> map = ScriptParser.parse(contentTypeStr);
			if (StringUtils.isNotEmpty(map.get(CHARSET)))
				charset = map.get(CHARSET);
		}

		String htmlSource;
		htmlSource = new String(bodybytes, charset);
		charset = HtmlCharacterReferenceUtil.getHtmlCharset(htmlSource);
		if (StringUtils.isNotBlank(charset)) {
			htmlSource = new String(bodybytes, charset);
			if (!charset.equalsIgnoreCase(source.getCharset())) {
				LogUtil.debug(logger, source.getPid(), " Source's charset changed");
				source.setCharset(charset);
			}
		}

		return htmlSource;
	}

	private Cookie[] getCookies(Source source) {
		String cookies = source.getCookies();
		return ELParser.parserCookies(cookies);
	}

	private NameValuePair[] getParameters(Source source) {
		String paramKeyValue = source.getParamKeyValue();
		return ELParser.parserParameters(paramKeyValue);
	}

}
